package website;

import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import org.w3c.dom.Document;
import page.Page;
import page.PageParserTool;
import page.RequestAndResponseTool;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.*;
import java.util.Iterator;
import java.util.List;
import java.util.Random;

public class TheLancet {

    private static String xmlPath = "D:\\paper_database\\Lancet0324\\%s.xml";
    private static int maxTime = 99;

    public void searchInAllIssuesPage(String allIssuesPageUrl) {
        if (allIssuesPageUrl == null) {
            allIssuesPageUrl = "https://www.thelancet.com/journals/lancet/issues#decade=loi_decade_201";
        }
        Page page = null;
        boolean isSuccessful = false;
        int ii = 0;
        do {
            if (ii++ > maxTime) {
                return;
            }
            try {
                Thread.sleep(new Random().nextInt(10) * 1000);
//                page = RequestAndResponseTool.sendRequstAndGetResponseByGet(allIssuesPageUrl);
                isSuccessful = true;
            } catch (Exception e) {
                e.printStackTrace();
            }
        } while (!isSuccessful);
        File file = new File("D://html.txt");
        BufferedReader bufferedReader = null;
        FileReader fileReader = null;
        StringBuilder sb = new StringBuilder();
        try {
            fileReader = new FileReader(file);
            bufferedReader = new BufferedReader(fileReader);
            String tempString = null;
            while ((tempString = bufferedReader.readLine()) != null) {
                sb.append(tempString);
            }
        } catch (Exception e) {

        } finally {
            try {
                fileReader.close();
                bufferedReader.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        page = new Page(sb.toString(), "", "");
        Elements es = PageParserTool.select(page,"a[class=issueLinkCon]");
        List<String> urlList = es.eachAttr("href");
        for (String url : urlList) {
            searchInOneIssues("https://www.thelancet.com" + url);
        }
    }

    public void searchInOneIssues(String issueUrl) {
        if (issueUrl == null) {
            issueUrl = "https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(19)32008-2/fulltext";
        }
        Page page = null;
        boolean isSuccessful = false;
        int ii = 0;
        do {
            if (ii++ > maxTime) {
                return;
            }
            try {
                Thread.sleep(new Random().nextInt(10) * 1000);
                page = RequestAndResponseTool.sendRequstAndGetResponseByGet(issueUrl);
                isSuccessful = true;
            } catch (Exception e) {
                e.printStackTrace();
            }
        } while (!isSuccessful);
        Elements es = PageParserTool.select(page,"section[class=toc__section]");
//        Elements es = PageParserTool.select(page,"section[class=toc__section] > h2:contains(Articles)");
//        Elements es = PageParserTool.select(page,"h3[class=toc__item__title]");
        for (Element element : es) {
            Elements select = element.select("h2:contains(Articles)");
            if (select.size() == 0) {
                continue;
            }
            Elements elements = element.select("h3[class=toc__item__title]");
            for (Element element1 : elements) {
                String title = element1.child(0).text();
                String href = element1.child(0).attr("href");
                String url = "https://www.thelancet.com" + href;
                downloadPaper(url);
            }

        }

    }

    public void downloadPaper(String paperUrl) {
        if (paperUrl == null) {
            paperUrl = "https://www.thelancet.com/journals/lancet/article/PIIS0140-6736(19)32008-2/fulltext";
        }
        Page page = null;
        boolean isSuccessful = false;
        int ii = 0;
        do {
            if (ii++ > maxTime) {
                return;
            }
            try {
                Thread.sleep(new Random().nextInt(10) * 1000);
                page = RequestAndResponseTool.sendRequstAndGetResponseByGet(paperUrl);
                isSuccessful = true;
            } catch (Exception e) {
                e.printStackTrace();
            }
        } while (!isSuccessful);
        Elements es = PageParserTool.select(page,"h1[class=article-header__title]");
        String title = es.text();
        es = PageParserTool.select(page,"a[title=Purchase one-time access]");
        String needPay = "false";
        if (es.size() != 0) {
            needPay = "true";
        }
        es = PageParserTool.select(page,"a[title=Register for free]");
        if (es.size() != 0) {
            needPay = "true";
        }
        es = PageParserTool.select(page,"a[class=loa__item__name article-header__info__ctrl loa__item__email]");
        String authors = "";
        String correspondence_author = "";
        int i = 0;
        for(Element element : es) {
            if (i++ != 0) {
                authors = authors.concat(";");
            }
            authors = authors.concat(element.text().trim());
//            String authorName = element.text().trim().split(",")[0];
//            String authorRank = element.text().trim().split(",")[1];
            if (element.childNodes().size() > 1) {
                correspondence_author = correspondence_author.concat(element.text().trim());
            }
        }
        es = PageParserTool.select(page,"span[class=article-header__journal]");
        String paper_type = es.text();
        es = PageParserTool.select(page,"span[class=article-header__publish-date__value]");
        String publish_date = es.text();
        // 正文
        es = PageParserTool.select(page,"div[class=col-md-7 col-lg-9 article__sections]");
        Elements sections = es.select("section");

//  参考文献
        es = PageParserTool.select(page,"li[class=ref]");

//        存xml
        String failingRecords = "D:\\failingrecords.txt";
        DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder builder = null;
        FileWriter fwriter = null;
        try {
            fwriter = new FileWriter(failingRecords);
            builder = documentBuilderFactory.newDocumentBuilder();
            Document document = builder.newDocument();
            //根
            org.w3c.dom.Element root = document.createElement("root");
            //todo: org.apache.xerces.dom
            //网址
            org.w3c.dom.Element urlElement = document.createElement("url");
            urlElement.setTextContent(paperUrl);
            root.appendChild(urlElement);
            //付费
            org.w3c.dom.Element payElement = document.createElement("needpay");
            payElement.setTextContent(needPay);
            root.appendChild(payElement);
            //时间
            org.w3c.dom.Element publicationTimeElement = document.createElement("publicationDate");
            publicationTimeElement.setTextContent(publish_date);
            root.appendChild(publicationTimeElement);
            //题目
            org.w3c.dom.Element titleElement = document.createElement("title");
            titleElement.setTextContent(title);
            root.appendChild(titleElement);
            //题目作者
            org.w3c.dom.Element authorElement = document.createElement("authors");
            authorElement.setTextContent(authors);
            root.appendChild(authorElement);
            //类型
            org.w3c.dom.Element typeElement = document.createElement("type");
            typeElement.setTextContent(paper_type);
            root.appendChild(typeElement);
            //正文
            org.w3c.dom.Element contentElement = document.createElement("content");
            org.w3c.dom.Element summaryElement = document.createElement("Summary");

            for (Element section : sections) {
                org.w3c.dom.Element parse = parse(document, section);
                if (parse != null) {
                    contentElement.appendChild(parse);
                }
            }
            root.appendChild(contentElement);

            //参考文献
            org.w3c.dom.Element referencesElement = document.createElement("references");
            if (es.size() == 0) {
                referencesElement.setTextContent("");
            } else {
                for (Element e : es) {
                    org.w3c.dom.Element referenceElement = document.createElement("reference");
                    String refString = "";
                    for (Node node : e.childNodes()) {
                        if (node.nodeName().equals("div") && node.attr("class").equals("ref__controls")) {
                            continue;   //下面两行View in Article 跳过
                        }
                        if (node.nodeName().equals("span") && node.attr("class").equals("refLabel")) {
                            referenceElement.setAttribute("label", ((Element) node).text());
                        } else if (node.nodeName().equals("ul") && node.attr("class").equals("rlist inline-bullet-list ref__authors ref__inline-block")) {
                            org.w3c.dom.Element authors1Element = document.createElement("authors");
                            authors1Element.setTextContent(((Element) node).text());
                            referenceElement.appendChild(authors1Element);
                        }else if (node.nodeName().equals("div") && node.attr("class").equals("ref__title")) {
                            org.w3c.dom.Element title1Element = document.createElement("title");
                            title1Element.setTextContent(((Element) node).text());
                            referenceElement.appendChild(title1Element);
                        } else {
                            try {
                                refString = refString.concat(((Element) node).text());
                            } catch (Exception e1) {
                                refString = refString.concat(((TextNode) node).text());
                            }
                        }
                    }
                    org.w3c.dom.Element hostElement = document.createElement("host");
                    hostElement.setTextContent(refString);
                    referenceElement.appendChild(hostElement);
                    referencesElement.appendChild(referenceElement);
                }
            }
            root.appendChild(referencesElement);

            document.appendChild(root);

            //创建转换工厂，然后将创建的document转换输出到文件中或控制台
            TransformerFactory transformerFactory = TransformerFactory.newInstance();
            Transformer transformer = transformerFactory.newTransformer();
            transformer.transform(new DOMSource(document), new StreamResult(String.format(xmlPath, paperUrl.substring(53,74))));

            //将document中的信息转换为字符串输出到控制台中
            StringWriter stringWriter = new StringWriter();
            transformer.transform(new DOMSource(document), new StreamResult(stringWriter));
            fwriter.close();
        } catch (Exception e) {
            try {
                fwriter.write(paperUrl  + "\r\n");
                fwriter.close();
            } catch (Exception e1) {
                e1.printStackTrace();
            }
            e.printStackTrace();
        }

//        存数据库
//        LancetPaper lancetPaper = new LancetPaper();
//        lancetPaper.setUrl(paperUrl);
//        lancetPaper.setAuthors(authors);
//        lancetPaper.setCorrespondenceAuthor(correspondence_author);
//        lancetPaper.setPaperType(paper_type);
//        lancetPaper.setPublishDate(publish_date);
//
//        LancetPaperMapper lancetPaperMapper = SpringUtil.getBean(LancetPaperMapper.class);
//        lancetPaperMapper.updatePaperByUrl(lancetPaper);

    }

    public void findPaperUrlFromDatabase() {
//        LancetPaperMapper lancetPaperMapper = SpringUtil.getBean(LancetPaperMapper.class);
//        List<LancetPaper> lancetPaperList = lancetPaperMapper.selectUrlNoXmlfile();
//        for (LancetPaper lancetPaper : lancetPaperList) {
//            this.downloadPaper(lancetPaper.getUrl());
//        }
    }


    public static org.w3c.dom.Element parse(Document document, Element element) throws Exception {
        if (element == null){
            return null;
        }
        org.w3c.dom.Element resultElement = document.createElement(element.tagName());
        Elements select = element.select("h2[class=top]");
        if (select.size() == 0) {
            return null;
        }
        org.w3c.dom.Element nodeElement = document.createElement("title");
        String title = select.get(0).attr("data-left-hand-nav");
        nodeElement.setTextContent(title);
        resultElement.appendChild(nodeElement);
        select = element.select("section > div[class=section-paragraph]");
        if (select.size() == 0) {
            return resultElement;
        }
        // TODO: 2021/3/26 get(0)是错的，不止一个
        for (Element element1 : select) {
            org.w3c.dom.Element element2 = parseSectionParagraph(document, element1);
            if (element2 != null) {
                resultElement.appendChild(element2);
            }
        }



//        Iterator<Attribute> iterator = element.attributes().iterator();
//        while (iterator.hasNext()) {
//            Attribute attribute = iterator.next();
//            resultElement.setAttribute(attribute.getKey(), attribute.getValue());
//        }
//        int size = element.childNodes().size();
//        for(int i = 0; i < size; ++i) {
//            Node node = element.childNodes().get(i);
//            if (node instanceof Element) {
//                resultElement.appendChild(parse(document, (Element) node));
//            } else {
//                try {
//                    org.w3c.dom.Element nodeElement = document.createElement("text");
//                    nodeElement.setTextContent(((TextNode) node).text().trim());
//                    resultElement.appendChild(nodeElement);
//                } catch (Exception e) {
//
//                }
//            }
//        }
        return resultElement;
    }

    public static org.w3c.dom.Element parseSectionParagraph(Document document, Element element) throws Exception {
        org.w3c.dom.Element resultElement = document.createElement("section");
        Iterator<Node> iterator = element.childNodes().iterator();
        while (iterator.hasNext()) {
            Node next = iterator.next();
            org.w3c.dom.Element element2 = null;
            if (next instanceof Element) {
                Element e = (Element) next;
                String tagName = e.tagName();
                if ("h3".equals(tagName)) {
                    element2 = document.createElement("sectionTitle");
                    element2.setTextContent(e.text());
                    resultElement.appendChild(element2);
                } else if ("div".equals(tagName)) {
                    String aClass = e.attr("class");
                    if ("dropBlock reference-citations".equals(aClass)) {
                        org.w3c.dom.Element supElement = document.createElement("sup");
                        Elements select = e.select("sup");
                        if (select.size() == 0) {
                            continue;
                        }
                        String text = select.get(0).text();
                        supElement.setTextContent(text);
                        resultElement.appendChild(supElement);
                    } else if ("floatDisplay".equals(aClass)) {

                    } else if ("accordion--textbox".equals(aClass)) {
                        Elements select = e.select("div[class=accordion__content]");
                        if (select.size() > 0) {
                            org.w3c.dom.Element element1 = parseSectionParagraph(document, select.get(0));
                            if (element1 != null) {
                                resultElement.appendChild(element1);
                            }
                        }
                    } else {
                        element2 = document.createElement("section");
                        element2.setTextContent(e.text());
                        resultElement.appendChild(element2);
                    }
                } else if ("a".equals(tagName)) {
                    if ("".equals(e.text())) {
                        continue;
                    }
                    org.w3c.dom.Element textElement = document.createElement("text");
                    textElement.setTextContent(e.text());
                    resultElement.appendChild(textElement);
                }
            } else if (next instanceof TextNode) {
                TextNode node = (TextNode) next;
                if ("".equals(node.text())) {
                    continue;
                }
                org.w3c.dom.Element textElement = document.createElement("text");
                textElement.setTextContent(node.text());
                resultElement.appendChild(textElement);
            }
        }
        return resultElement;
    }

    public void updateXMLFile () {
//        LancetPaperMapper lancetPaperMapper = SpringUtil.getBean(LancetPaperMapper.class);
//        List<LancetPaper> lancetPaperList = lancetPaperMapper.selectAllUrl();
//        for (LancetPaper lancetPaper : lancetPaperList) {
//            String xmlFileName = lancetPaper.getUrl().substring(53,74) + ".xml";
//            lancetPaper.setXmlfile(xmlFileName);
//            lancetPaperMapper.updatePaperByUrl(lancetPaper);
//        }
    }

    public static void main(String[] args) {
        TheLancet theLancet = new TheLancet();
        theLancet.searchInAllIssuesPage(null);
//        theLancet.downloadPaper(null);
    }

}
